package edu.unc.ils.memai.extract;

import java.io.File;


import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

import maui.stemmers.PorterStemmer;
import maui.stemmers.Stemmer;
import maui.util.Candidate;

import org.apache.commons.io.FileUtils;

import edu.unc.ils.memai.tokenize.MauiTokenizer;
import edu.unc.ils.memai.tokenize.Tokenizer;
import edu.unc.ils.memai.vocab.LuceneVocabulary;
import edu.unc.ils.memai.vocab.Vocabulary;
import edu.unc.ils.util.IndexerConsistency;
import edu.unc.ils.util.ValueComparator;

public class CandidateExtractorTest 
{
   public static void main(String[] args) throws Exception {
        
        //Vocabulary vocab = new MauiVocabulary();
        Vocabulary vocab = new LuceneVocabulary();
        
        @SuppressWarnings("unchecked")
        List<String> stopwords = FileUtils.readLines(new File("src/main/resources/stopwords_en.txt"), "UTF-8");
        
        Stemmer stemmer = new PorterStemmer();
        vocab.setStemmer(stemmer);
        vocab.setStopwords(stopwords);
        vocab.setThreshold(10);
        vocab.setLowerCase(true);
        vocab.load("/Users/cwillis/dev/hive/hive-data/nalt/naltStoreEn");
        
        Tokenizer tok = new MauiTokenizer();
        
        CandidateExtractor extractor = new MauiCandidateExtractor();
        //CandidateExtractor extractor = new NASACandidateExtractor();
        extractor.setMaxPhraseLength(5);
        extractor.setMinPhraseLength(1);
        extractor.setMinOccurFrequency(1);
        extractor.setVocabulary(vocab);
        extractor.setStopwords(stopwords);
        
        extractor.setStemmer(stemmer);
     
        File dir = new File("/Users/cwillis/Desktop/gold_set/agricola/abstract");
        File[] files = dir.listFiles();
        DecimalFormat df = new DecimalFormat("##.#####");
        for (File file: files)
        {
            String path = file.getAbsolutePath();
            if (file.isFile() & path.endsWith(".txt"))
            {
                String name = file.getName();
                String id = name.substring(0, name.indexOf("."));
                String prefix = path.substring(0, path.lastIndexOf("."));
                String text = FileUtils.readFileToString(new File(path), "UTF-8");
                
                @SuppressWarnings("unchecked")
                List<String> headers = FileUtils.readLines(new File(prefix + ".subjects"), "UTF-8");
                for (int i=0; i< headers.size(); i++)
                {
                    headers.set(i, headers.get(i).toLowerCase());
                }
                
                List<String> candidates = getCandidates(text, extractor, stemmer, vocab, tok);
                for (int i=0; i< candidates.size(); i++)
                {
                    candidates.set(i, candidates.get(i).toLowerCase());
                }
                double p = IndexerConsistency.precision(headers, candidates);
                double r = IndexerConsistency.recall(headers, candidates);
                
                List<String> top = candidates;
                if (candidates.size() > 20)
                    top = candidates.subList(0, 20);
                
                double p2 = IndexerConsistency.precision(headers, top);
                double r2 = IndexerConsistency.recall(headers, top);
                
                System.out.println(id + "|" + df.format(p) + "|" +  df.format(r) + "|" + candidates.size() + "|" + p2 + "|" + r2 + "|20");
                
            }
        }
   }
   
        
   public static List<String> getCandidates(String text, CandidateExtractor extractor,
           Stemmer stemmer, Vocabulary vocab, Tokenizer tok) throws Exception
   {
       List<String> list = new ArrayList<String>();
       
       List<String> segments = tok.tokenize(text, false);

       Map<String, Candidate> candidates = extractor.getCandidates(segments);
       Map<String, Integer> candidateFreq = new HashMap<String, Integer>();
       for (String candidateId: candidates.keySet())
       {
           Candidate candidate = candidates.get(candidateId);
           double frequency = candidate.getFrequency();
           candidateFreq.put(candidate.getTitle(), (int)frequency);
           //System.out.println(candidate.getTitle() + "|" + candidate.getFrequency());
       }
       
       
       ValueComparator vc = new ValueComparator(candidateFreq);
       Map<String, Integer> sortedMap = new TreeMap<String, Integer>(vc);
       sortedMap.putAll(candidateFreq);
       for (String name: sortedMap.keySet()) {
           list.add(name);
       }
       return list;
    }

    /*
    String text = "Background: Cotton (Gossypium hirsutum L.) fibers are trichomes that initiate from the ovule epidermis. " + 
    "Little is known about the developmental pathway causing fiber to differentiate from ovular epidermal cells even " + 
    "though limits on the number of cells that differentiate into fiber will limit yield. Results: A method was developed " + 
    "to isolate RNA from fiber initials 1 day post anthesis (dpa). Complementary DNA libraries representing 1 dpa fibers " + 
    "and other cotton tissues were sequenced and analyzed. Assembly of G. hirsutum Expressed Sequenced Tags (ESTs) identified " + 
    "over 11,000 sequences not previously represented in GenBank. New genes identified among these ESTs were represented " + 
    "on microarrays. The microarrays were used to identify genes enriched in fiber initials (1 dpa fibers) and elongating " + 
    "fibers. Analyses of Gene Ontologies (GO) of differentially expressed genes determined that terms associated with the " + 
    "membranes were statistically over represented among genes increased in expression in fiber initials and 10 dpa fibers. " + 
    "Staining ovules with a fluorescent dye confirmed an increase in Endoplasmic Reticulum (ER) occurred in fiber initials on " + 
    "the day of anthesis, persisted through 3 dpa and was absent in a fiberless mutant. Two genes similar to the " + 
    "CAPRICE/TRIPTYCHON (CPC) gene that inhibits differentiation of leaf trichomes in Arabidopsis were also characterized. " + 
    "Genes associated with novel regulation of brassinosterols, GTP mediated signal transduction and cell cycle control and " +
    "components of a Ca+2 mediated signaling pathway were identified. Staining of cellular Ca+2 indicated that fiber initials " +
    "had more Ca+2 than other ovule cells supporting a role for Ca+2 in fiber development. Conclusion: Analysis of genes " + 
    "expressed in fiber initials identified a unique stage in fiber development characterized by an increase in ER and Ca+2 " + 
    "levels that occurred between 0 and 1 dpa. The gene similar to CPC has a MYB domain but appears to lack a transcription " + 
    "activating domain similar to the Arabisopsis gene. The method used to stain the ER also can be used to count fiber " + 
    "initials and showed fiber cells develop from adjacent cells unlike leaf trichomes.";
    */
}
